package com.zbl.service;

import com.gargoylesoftware.htmlunit.*;
import com.gargoylesoftware.htmlunit.html.*;
import com.xuxueli.crawler.XxlCrawler;
import com.xuxueli.crawler.parser.PageParser;
import com.xuxueli.crawler.parser.strategy.NonPageParser;
import com.zbl.dto.Novel;
import com.zbl.parser.JsoupParser;
import com.zbl.parser.TableJsoupParse;
import com.zbl.vo.NovelVo;
import com.zbl.vo.WrapperRequestParam;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.io.IOException;
import java.util.List;
import java.util.Objects;

/**
 * @Author: zhangjun
 * @Date: 2020/7/29 14:43
 * @Description: 爬取多页数据
 */
@Service
public class MultiPageCrawlerService {
    @Resource
    private MongoTemplate mongoTemplate;

    /**
     * xxl 方式爬取多页数据
     * @param url
     */
    public void recursionCrawler(String url) {
        XxlCrawler crawler = new XxlCrawler.Builder()
                .setUrls(url)
                .setWhiteUrlRegexs(url)
                .setThreadCount(1)
                .setPageParser(new PageParser<NovelVo>() {
                    @Override
                    public void parse(Document html, Element pageVoElement, NovelVo novelVo) {
                        // 解析封装 PageVo 对象
                        String pageUrl = html.baseUri();
                        System.out.println(pageUrl + "：" + novelVo.toString());


                        System.out.println(">>>>>>>>>>>>>>>> " + pageVoElement);
                        System.out.println(">>>>>>>>>>>>>>>> " + html.getElementsByClass("nextChapter"));

                        //数据存储到
                        Novel novel = new Novel();
                        novel.setChapter(novelVo.getChapter());
                        novel.setContent(novelVo.getContent());
                        mongoTemplate.save(novel);

                        if (Objects.nonNull(novelVo.getNextUrl())) {
                            recursionCrawler(novelVo.getNextUrl());
                        }
                    }
                })
                .build();
        crawler.start(true);
    }

    /**
     * webclient方式 爬取多页数据
     * @param baseUrl
     * @throws IOException
     */
    public void webClintCrawlerData(String baseUrl) throws IOException {
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        //ajax
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
        //支持js
        webClient.getOptions().setJavaScriptEnabled(true);
        //忽略js错误
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        //忽略css错误
        webClient.setCssErrorHandler(new SilentCssErrorHandler());
        //不执行CSS渲染
        webClient.getOptions().setCssEnabled(false);

        //允许重定向
        webClient.getOptions().setRedirectEnabled(true);
        //允许cookie
        webClient.getCookieManager().setCookiesEnabled(true);

        //指定是否在出现故障状态代码时抛出异常
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        //启用/禁用“不跟踪”支持
        webClient.getOptions().setDoNotTrackEnabled(false);
        //如果设置为true，客户机将接受与任何主机的连接，而不管它们是否有有效证书
        webClient.getOptions().setUseInsecureSSL(true);

        //设置该程序的超时时间。
        webClient.getOptions().setTimeout(30000);
        //设置js执行的超时时间。
        webClient.setJavaScriptTimeout(30000);
        //异步JS执行需要耗时,所以这里线程要阻塞xx秒,等待异步JS执行结束
        webClient.waitForBackgroundJavaScript(15000);

        HtmlPage htmlPage = webClient.getPage(baseUrl);
//        System.out.println(htmlPage.asXml());

        /*HtmlSubmitInput btnQuery = (HtmlSubmitInput) htmlPage.getElementById("Button1");
        HtmlPage result = btnQuery.click();*/

//        HtmlTable rows = (HtmlTable) htmlPage.getElementsByTagName("table");
        HtmlTable table = (HtmlTable) htmlPage.getElementsByTagName("table").get(2);
        List<HtmlTableRow> rows = table.getRows();
        for (int i=1;i<rows.size(); i++) {
            System.out.printf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n",
                    rows.get(i).getCell(0).getFirstElementChild().asText(),
                    rows.get(i).getCell(1).asText(),
                    rows.get(i).getCell(2).asText(),
                    rows.get(i).getCell(3).asText(),
                    rows.get(i).getCell(4).asText(),
                    rows.get(i).getCell(5).asText(),
                    rows.get(i).getCell(6).asText());
        }
    }

    /**
     * webclient方式 爬取多页数据 使用jsoup方式解析
     * @param param
     * @throws IOException
     */
    public void webClintCrawlerDataParseWithJSoup(WrapperRequestParam param, NonPageParser pageParser) throws IOException {
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        //ajax
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
        //支持js
        webClient.getOptions().setJavaScriptEnabled(true);
        //忽略js错误
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        //忽略css错误
        webClient.setCssErrorHandler(new SilentCssErrorHandler());
        //不执行CSS渲染
        webClient.getOptions().setCssEnabled(false);

        //允许重定向
        webClient.getOptions().setRedirectEnabled(true);
        //允许cookie
        webClient.getCookieManager().setCookiesEnabled(true);

        //指定是否在出现故障状态代码时抛出异常
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        //启用/禁用“不跟踪”支持
        webClient.getOptions().setDoNotTrackEnabled(false);
        //如果设置为true，客户机将接受与任何主机的连接，而不管它们是否有有效证书
        webClient.getOptions().setUseInsecureSSL(true);

        //设置该程序的超时时间。
        webClient.getOptions().setTimeout(30000);
        //设置js执行的超时时间。
        webClient.setJavaScriptTimeout(30000);
        //异步JS执行需要耗时,所以这里线程要阻塞xx秒,等待异步JS执行结束
        webClient.waitForBackgroundJavaScript(150000);

        HtmlPage htmlPage = webClient.getPage(param.getUrl());

        /*HtmlSubmitInput btnQuery = (HtmlSubmitInput) htmlPage.getElementById("Button1");
        HtmlPage result = btnQuery.click();*/

        //Jsoup解析html
        pageParser.parse(param.getUrl(), htmlPage.asXml());
    }
}
